package com.chance.cc.crawler.development.bootstrap.du;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.queue.crawler.CrawlerQueueConfig;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;

import java.util.*;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

public class DuappStart {

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static final String metaSite = "product";
    public static final String startUrl = "api/v1/h5/commodity/fire/search/doCategoryDetail";
    public static final String domain = "du_app";

    private static final String domainId = "du_app";

    public static void main(String[] args) {

//        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
//                .startPageRequest(domainId, turnPage)
//                .httpUrl("/aweme/v1/general/search/single")
//                .releaseTime(System.currentTimeMillis())
//                .filter(CrawlerEnum.CrawlerRecordFilter.key)
//                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
//                .needWashed(false)
//                .needParsed(false)
//                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);
//
//        RedisReader redisReader = new RedisReader("192.168.1.215",6379,2);
//        CrawlerRequestRecord crawlerRequestRecord = redisReader.recordPop("crawler-du_app-queue_list", "crawler-du_app-queue_map");
//
//        DevCrawlerController.builder()
//                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
//                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
//                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\log\\du\\du_hot_news.json",false)
//                .requestRecord(crawlerRequestRecord)
//                .build()
//                .start();

//        testProduct();
        crawlerScheduler();
//        askRecordTest();
    }


    public static void testProduct(){
        FilterInfo turnPageFilterInfo = new FilterInfo();
        turnPageFilterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        turnPageFilterInfo.setCountTotalNum(100);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, turnPage)
                .httpUrl("api/v1/h5/commodity/fire/search/doCategoryDetail#file")
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo("du_app"))
                .needWashed(false)
                .needParsed(false)
                .turnPageFilterInfo(turnPageFilterInfo)
                .proxy(proxy)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("product");
        requestRecord.tagsCreator().bizTags().addCustomKV("crawler_type","findList");
        requestRecord.tagsCreator().bizTags().addCustomKV("filePath","C:\\Users\\Administrator\\Documents\\chance\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\du\\urls.json");

        //初始 comment crawler request record
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("du_app",turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/"+domainId+"/keys?site=product")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline("console") //控制台输t出
                .fileResultPipeline("file_add_result","D:\\chance\\log\\du\\du_product_add.json",false)
                .fileResultPipeline("file_del_result","D:\\chance\\log\\du\\du_product_del.json",false)
                .requestRecord(requestRecord)
//                .supportRecord(initNewsCrawlerRecord)
                .crawlerThreadNum(20)
                .build()
                .start();
    }


    public static void crawlerScheduler(){

        String requestQueueName = StringUtils.joinWith("-", "crawler", domain, metaSite,"new", "productDetail", "queue");
        //文章采集
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        CrawlerRequestRecord articleCrawler = redisReader.recordPop(requestQueueName+"_list",requestQueueName+"_map");

        //配置结果队列信息
        String resultDelQueueName = "delResult-du_app-spuIds";
        CrawlerQueueConfig redisDelResultQueue = new CrawlerQueueConfig(
                resultDelQueueName,
                CrawlerQueueConfig.Content.result,
                CrawlerQueueConfig.Storage.redis,
                CrawlerQueueConfig.Structure.list);

        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline("console") //控制台输t出
//                .fileResultPipeline("file_add_result","D:\\chance\\log\\du\\du_product_add.json",false)
//                .fileResultPipeline("file_del_result","D:\\chance\\log\\du\\du_product_del.json",false)
                .requestRecord(articleCrawler)
//                .supportRecord(initNewsCrawlerRecord)
                .crawlerThreadNum(1)
                .build("com.chance.cc.crawler.development.scripts.du")
                .start();



    }


    /**
     * 初始record
     * @return
     */
    public static CrawlerRequestRecord doSearchNewsCrawler(){

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPage)
                .httpUrl(startUrl)
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.key)
                .addFilterInfo(FilterUtils.redisFilterKeyInfo("filter-du_app-spuIds"))
                .proxy(proxy)
                .build();

        requestRecord.getHttpConfig().setProxyQueueSize(1);
        requestRecord.getHttpConfig().setProxyType("queue");

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite(metaSite);
        requestRecord.tagsCreator().bizTags().addSiteBiz(metaSite);
        requestRecord.tagsCreator().bizTags().addCustomKV("crawler_type","productDetail");
        return requestRecord;
    }

    public static void testCalender(){

        HttpConfig me = HttpConfig.me(domainId);
        me.setHttpSite("sell_calender");
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, turnPage)
                .httpUrl("https://m.dewu.com/sns/v1/sell/index?v=4.2.0")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(me)
                .needWashed(false)
                .needParsed(true)
                .proxy(proxy)
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domainId);
        requestRecord.tagsCreator().bizTags().addSite("sell_calender");
        requestRecord.tagsCreator().bizTags().addSiteBiz("sell_calender");

        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
                .fileResultPipeline("F:\\chance_log\\du\\du_sell_calender.json",false)
                .requestRecord(requestRecord)
                .crawlerThreadNum(20)
                .build("com.chance.cc.crawler.development.scripts.du")
                .start();
    }

    public static void askRecordTest(){
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, turnPage)
                .httpRequest(getAskRequest())
                .releaseTime(System.currentTimeMillis())
                .needWashed(true)
//                .needParsed(false)
                .proxy(proxy)
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domainId);
        requestRecord.tagsCreator().bizTags().addSite("sell_calender");
        requestRecord.tagsCreator().bizTags().addSiteBiz("sell_calender");
        requestRecord.tagsCreator().bizTags().addCustomKV("product_id","123");
        requestRecord.tagsCreator().bizTags().addCustomKV("product_detail","1243");
        requestRecord.tagsCreator().bizTags().addCustomKV("sell_product","123");

        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\log\\du\\du_sell_calender.json",false)
                .requestRecord(requestRecord)
//                .crawlerThreadNum(20)
                .build()
                .start();

    }

    public static HttpRequest getAskRequest(){
        String url = "https://app.dewu.com/api/v1/h5/newbidding/buyer/view";
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(url);
        httpRequest.addHeader("appid","wxapp");
        httpRequest.addHeader("content-type","application/json");
        httpRequest.setMethod("POST");
        Map<String,Object> params = new HashMap<>();
        params.put("spuId","1714743");

        String sign = genSignature(params);
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);
        httpRequest.addHeader("Host","app.dewu.com");
        httpRequest.addHeader("Connection","keep-alive");
        httpRequest.addHeader("User-Agent",genRandomUA());
        httpRequest.addHeader("Content-Type","application/json");
        httpRequest.addHeader("platform","h5");
        httpRequest.addHeader("AppId","wxapp");
        httpRequest.addHeader("appVersion","4.4.0");
        httpRequest.addHeader("charset","utf-8");
        httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
        httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
        httpRequest.addHeader("wxapp-login-token","");
        return httpRequest;
    }

    public static String genSignature(Map<String,Object> params) {
        return genSignature(params,"19bc545a393a25177083d4a748807cc0");
    }

    public static String genSignature(Map<String,Object> params,String salt) {
        if(params == null){
            return null;
        }
        String sign = "";
        StringBuffer sb = new StringBuffer();
        try {
            // 1. 字典升序排序
            SortedMap<String,Object> sortedMap = new TreeMap<>(params);
            // 2. 拼按URL键值对
            Set<String> keySet = sortedMap.keySet();
            for(String key : keySet){
                //sign不参与算法
                if(key.equals("sign")){
                    continue;
                }

                Object obj =  sortedMap.get(key);
                String value = obj.toString() ;
                if (obj instanceof Map){
                    value = JSON.toJSONString(obj);
                }
                if (obj instanceof List){
                    List list = (List) obj;
                    value = "";
                    for (Object o : list) {
                        if (value.length() > 0){
                            value += ",";
                        }
                        value += JSON.toJSONString(o);
                    }
//                    value = ","+JSON.toJSONString(obj);
                }
//                if ("[]".equals(value)){
//                    value = "";
//                }
                sb.append(key).append(value);

            }
            String uriString = sb.toString() + salt;
            // 3. MD5运算得到请求签名
            sign = md5(uriString);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sign;
    }

    private static String genRandomUA(){
        int part1 = RandomUtils.nextInt(1000, 9999);
        int part2 = RandomUtils.nextInt(1000, 1999);
        int part3 = RandomUtils.nextInt(100, 999);

        return String.format("Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2797 MMWEBSDK/20210501 Mobile Safari/537.36 MMWEBID/%s MicroMessenger/8.0.6.%s(0x28000%s) Process/appbrand0 WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64 MiniProgramEnv/android",
                part1,part2,part3);
    }

}
